lines = readLines(paste0(RESEARCH_HOME, PATH_FILES, NOHUP_FILE))
lines = lines[grepl("\\|[1-6]\\.", lines)]
ff = as_tibble(lines) %>%
  separate(value, into=c("Timestamp", "Title", "ID", "Nodes", "Cores", "Status", "Time", "Stage", "Duration", "Load", "Interval"), sep="\\|") %>%
  separate(ID, into=c(NA, NA, "ID"), sep="-") %>%
  mutate(ID = as.numeric(ID), Stage = str_trim(Stage), Duration = as.numeric(Duration)) %>%
  mutate(uStart = parse_datetime(str_replace(Timestamp, ",", ".")), FFStage = Stage, Status = str_trim(Status)) %>%
  select(uStart, Nodes, Stage, Duration, Interval, FFStage, Status)
head(ff)
stageFF = ff %>% filter(str_trim(Status) == "END") %>% 
  mutate(Stage = paste0(Interval,".",Stage)) %>% 
  select(Nodes, Stage, Duration)
head(stageFF)
p = ggplot(data = stageFF, aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(x="Stages", y="Duration(s)")
ggplotly(p)
stages = rbind(stages0, ff) %>% arrange(uStart)
head(stages, n=100)
n = nrow(stages)
ffstage  = ""
interval = ""
for(row in 1:n){
  if(stages[row, "Status"] == "START"){
    ffstage  = stages[row, "FFStage"]
    interval = stages[row, "Interval"]
  } else {
    if(stages[row, "Status"] == "END"){
      ffstage  = ""
      interval = ""
    } else {
      stages[row, "FFStage"] = ffstage
      stages[row, "Interval"] = interval
    }
  }
}
head(stages, n=100)
s = stages %>% filter(Interval != "") %>% filter(grepl("_", Stage)) %>% 
  mutate(FFStage = paste0(Interval,".",FFStage)) %>% 
  select(Nodes, Stage, Duration, FFStage, Interval)
head(s)
interval = 0
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 1
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 2
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 3
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 4
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKCmBgYHtyfQpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeShsdWJyaWRhdGUpCmxpYnJhcnkocGxvdGx5KQoKUkVTRUFSQ0hfSE9NRSA9ICIvaG9tZS9hbmQvRG9jdW1lbnRzL1BoRC9SZXNlYXJjaCIKUEFUSF9GSUxFUyA9ICIvU2NyaXB0cy9SL0JlbmNobWFya3MvTXVsdGlBbmRTaW5nbGVOb2RlL1IxMi9kYmxhYi8iCk5PSFVQX0ZJTEUgPSAibm9odXAxMC50c3YiCmxpbmVzID0gcmVhZExpbmVzKHBhc3RlMChSRVNFQVJDSF9IT01FLCBQQVRIX0ZJTEVTLCBOT0hVUF9GSUxFKSkKbGluZXMgPSBsaW5lc1tncmVwbCgiXFx8U1RBR0VTXFx8IiwgbGluZXMpXQpzdGFnZXMwID0gYXNfdGliYmxlKGxpbmVzKSAlPiUKICBzZXBhcmF0ZSh2YWx1ZSwgaW50bz1jKCJUaW1lc3RhbXAiLCAiVGl0bGUiLCAiTm9kZXMiLCAiQ29yZXMiLCJTdGFnZUlEIiwgIlN0YWdlIiwgIlN0YXJ0IiwgIkVuZCIsICJuVGFza3MiLCAiUnVudGltZSIsICJDUFVUaW1lIiwgIklucHV0Qnl0ZXMiLCAiSW5wdXRSZWNvcmRzIiwgIlNodWZmbGVCeXRlcyIsICJTaHVmZmxlUmVjb3JkcyIsICJJRCIpLCBzZXA9IlxcfCIpICU+JQogIHNlcGFyYXRlKElELCBpbnRvPWMoTkEsIE5BLCAiSUQiKSwgc2VwPSItIikgJT4lCiAgbXV0YXRlKElEID0gYXMubnVtZXJpYyhJRCksIFN0YWdlID0gc3RyX3RyaW0oU3RhZ2UpKSAlPiUKICBtdXRhdGUodVN0YXJ0ID0gcGFyc2VfZGF0ZXRpbWUoc3RyX3JlcGxhY2Uoc3RyX3RyaW0oU3RhcnQpLCAiR01UIiwgIiIpKSwgCiAgICAgICAgIHVFbmQgPSBwYXJzZV9kYXRldGltZShzdHJfcmVwbGFjZShzdHJfdHJpbShFbmQpLCAiR01UIiwgIiIpKSkgJT4lCiAgbXV0YXRlKER1cmF0aW9uID0gYXMubnVtZXJpYyh1RW5kIC0gdVN0YXJ0KSwgCiAgICAgICAgIHVTdGFydCA9IHVTdGFydCAtIGhvdXJzKDcpLAogICAgICAgICBTdGFnZSA9IHBhc3RlMChzdHJfcGFkKHN0cl90cmltKFN0YWdlSUQpLCBwYWQ9IjAiLCBzaWRlPSJsZWZ0Iiwgd2lkdGg9NSksIl8iLFN0YWdlKSkgJT4lCiAgc2VsZWN0KHVTdGFydCwgTm9kZXMsIFN0YWdlLCBEdXJhdGlvbikgJT4lCiAgYXJyYW5nZSh1U3RhcnQpCnN0YWdlczAkSW50ZXJ2YWwgPSAtMgpzdGFnZXMwJEZGU3RhZ2UgPSAiIgpzdGFnZXMwJFN0YXR1cyA9ICIiCmhlYWQoc3RhZ2VzMCwgbj0xNSkKYGBgCgpgYGB7cn0KbGluZXMgPSByZWFkTGluZXMocGFzdGUwKFJFU0VBUkNIX0hPTUUsIFBBVEhfRklMRVMsIE5PSFVQX0ZJTEUpKQpsaW5lcyA9IGxpbmVzW2dyZXBsKCJcXHxbMS02XVxcLiIsIGxpbmVzKV0KZmYgPSBhc190aWJibGUobGluZXMpICU+JQogIHNlcGFyYXRlKHZhbHVlLCBpbnRvPWMoIlRpbWVzdGFtcCIsICJUaXRsZSIsICJJRCIsICJOb2RlcyIsICJDb3JlcyIsICJTdGF0dXMiLCAiVGltZSIsICJTdGFnZSIsICJEdXJhdGlvbiIsICJMb2FkIiwgIkludGVydmFsIiksIHNlcD0iXFx8IikgJT4lCiAgc2VwYXJhdGUoSUQsIGludG89YyhOQSwgTkEsICJJRCIpLCBzZXA9Ii0iKSAlPiUKICBtdXRhdGUoSUQgPSBhcy5udW1lcmljKElEKSwgU3RhZ2UgPSBzdHJfdHJpbShTdGFnZSksIER1cmF0aW9uID0gYXMubnVtZXJpYyhEdXJhdGlvbikpICU+JQogIG11dGF0ZSh1U3RhcnQgPSBwYXJzZV9kYXRldGltZShzdHJfcmVwbGFjZShUaW1lc3RhbXAsICIsIiwgIi4iKSksIEZGU3RhZ2UgPSBTdGFnZSwgU3RhdHVzID0gc3RyX3RyaW0oU3RhdHVzKSkgJT4lCiAgc2VsZWN0KHVTdGFydCwgTm9kZXMsIFN0YWdlLCBEdXJhdGlvbiwgSW50ZXJ2YWwsIEZGU3RhZ2UsIFN0YXR1cykKCmhlYWQoZmYpCmBgYAoKYGBge3J9CnN0YWdlRkYgPSBmZiAlPiUgZmlsdGVyKHN0cl90cmltKFN0YXR1cykgPT0gIkVORCIpICU+JSAKICBtdXRhdGUoU3RhZ2UgPSBwYXN0ZTAoSW50ZXJ2YWwsIi4iLFN0YWdlKSkgJT4lIAogIHNlbGVjdChOb2RlcywgU3RhZ2UsIER1cmF0aW9uKQpoZWFkKHN0YWdlRkYpCgpwID0gZ2dwbG90KGRhdGEgPSBzdGFnZUZGLCBhZXMoeCA9IFN0YWdlLCB5ID0gRHVyYXRpb24sIGZpbGwgPSBOb2RlcykpICsKICBnZW9tX2JhcihzdGF0PSJpZGVudGl0eSIsIHBvc2l0aW9uPXBvc2l0aW9uX2RvZGdlKHdpZHRoID0gMC43NSksIHdpZHRoID0gMC43KSArIAogIHRoZW1lKGF4aXMudGV4dC54ID0gZWxlbWVudF90ZXh0KGFuZ2xlID0gOTAsIGhqdXN0ID0gMSkpICsKICBsYWJzKHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKYGBgCgoKYGBge3J9CnN0YWdlcyA9IHJiaW5kKHN0YWdlczAsIGZmKSAlPiUgYXJyYW5nZSh1U3RhcnQpCgpoZWFkKHN0YWdlcywgbj0xMDApCmBgYAoKYGBge3J9Cm4gPSBucm93KHN0YWdlcykKZmZzdGFnZSAgPSAiIgppbnRlcnZhbCA9ICIiCmZvcihyb3cgaW4gMTpuKXsKICBpZihzdGFnZXNbcm93LCAiU3RhdHVzIl0gPT0gIlNUQVJUIil7CiAgICBmZnN0YWdlICA9IHN0YWdlc1tyb3csICJGRlN0YWdlIl0KICAgIGludGVydmFsID0gc3RhZ2VzW3JvdywgIkludGVydmFsIl0KICB9IGVsc2UgewogICAgaWYoc3RhZ2VzW3JvdywgIlN0YXR1cyJdID09ICJFTkQiKXsKICAgICAgZmZzdGFnZSAgPSAiIgogICAgICBpbnRlcnZhbCA9ICIiCiAgICB9IGVsc2UgewogICAgICBzdGFnZXNbcm93LCAiRkZTdGFnZSJdID0gZmZzdGFnZQogICAgICBzdGFnZXNbcm93LCAiSW50ZXJ2YWwiXSA9IGludGVydmFsCiAgICB9CiAgfQp9CgpoZWFkKHN0YWdlcywgbj0xMDApCmBgYAoKYGBge3J9CnMgPSBzdGFnZXMgJT4lIGZpbHRlcihJbnRlcnZhbCAhPSAiIikgJT4lIGZpbHRlcihncmVwbCgiXyIsIFN0YWdlKSkgJT4lIAogIG11dGF0ZShGRlN0YWdlID0gcGFzdGUwKEludGVydmFsLCIuIixGRlN0YWdlKSkgJT4lIAogIHNlbGVjdChOb2RlcywgU3RhZ2UsIER1cmF0aW9uLCBGRlN0YWdlLCBJbnRlcnZhbCkKCmhlYWQocykKYGBgCgpgYGB7cn0KaW50ZXJ2YWwgPSAwCnAgPSBnZ3Bsb3QoZGF0YSA9IHMgJT4lIGZpbHRlcihJbnRlcnZhbCA9PSBpbnRlcnZhbCksIGFlcyh4ID0gU3RhZ2UsIHkgPSBEdXJhdGlvbiwgZmlsbCA9IE5vZGVzKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGggPSAwLjc1KSwgd2lkdGggPSAwLjcpICsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkgKwogIGxhYnModGl0bGU9cGFzdGUoIlRpbWUgaW50ZXJ2YWwiLCBpbnRlcnZhbCksIHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKaW50ZXJ2YWwgPSAxCnAgPSBnZ3Bsb3QoZGF0YSA9IHMgJT4lIGZpbHRlcihJbnRlcnZhbCA9PSBpbnRlcnZhbCksIGFlcyh4ID0gU3RhZ2UsIHkgPSBEdXJhdGlvbiwgZmlsbCA9IE5vZGVzKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGggPSAwLjc1KSwgd2lkdGggPSAwLjcpICsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkgKwogIGxhYnModGl0bGU9cGFzdGUoIlRpbWUgaW50ZXJ2YWwiLCBpbnRlcnZhbCksIHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKaW50ZXJ2YWwgPSAyCnAgPSBnZ3Bsb3QoZGF0YSA9IHMgJT4lIGZpbHRlcihJbnRlcnZhbCA9PSBpbnRlcnZhbCksIGFlcyh4ID0gU3RhZ2UsIHkgPSBEdXJhdGlvbiwgZmlsbCA9IE5vZGVzKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGggPSAwLjc1KSwgd2lkdGggPSAwLjcpICsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkgKwogIGxhYnModGl0bGU9cGFzdGUoIlRpbWUgaW50ZXJ2YWwiLCBpbnRlcnZhbCksIHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKaW50ZXJ2YWwgPSAzCnAgPSBnZ3Bsb3QoZGF0YSA9IHMgJT4lIGZpbHRlcihJbnRlcnZhbCA9PSBpbnRlcnZhbCksIGFlcyh4ID0gU3RhZ2UsIHkgPSBEdXJhdGlvbiwgZmlsbCA9IE5vZGVzKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGggPSAwLjc1KSwgd2lkdGggPSAwLjcpICsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkgKwogIGxhYnModGl0bGU9cGFzdGUoIlRpbWUgaW50ZXJ2YWwiLCBpbnRlcnZhbCksIHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKaW50ZXJ2YWwgPSA0CnAgPSBnZ3Bsb3QoZGF0YSA9IHMgJT4lIGZpbHRlcihJbnRlcnZhbCA9PSBpbnRlcnZhbCksIGFlcyh4ID0gU3RhZ2UsIHkgPSBEdXJhdGlvbiwgZmlsbCA9IE5vZGVzKSkgKwogIGdlb21fYmFyKHN0YXQ9ImlkZW50aXR5IiwgcG9zaXRpb249cG9zaXRpb25fZG9kZ2Uod2lkdGggPSAwLjc1KSwgd2lkdGggPSAwLjcpICsgCiAgdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA5MCwgaGp1c3QgPSAxKSkgKwogIGxhYnModGl0bGU9cGFzdGUoIlRpbWUgaW50ZXJ2YWwiLCBpbnRlcnZhbCksIHg9IlN0YWdlcyIsIHk9IkR1cmF0aW9uKHMpIikKZ2dwbG90bHkocCkKYGBgCg==